from sql_helper import MySqlHelper
import requests
import json
from selenium import webdriver

def read_list(url,all_list=None):
    if all_list is None:
        all_list = []

        page = range(1, 68)
        # for i in page:
        #     print(i)
        a = 0
        for i in page:
            if requests.get(url=url+str(i)).status_code == 200:
                response = json.loads(requests.get(url=url+str(i)).text)
                num = 0
                a = a + 1
                if 'subjects' in response['res']:
                    if 'title' in response['res']['payload']:
                        for i in response['res']['subjects']:
                            num = num + 1
                            l = []
                            l.append(num)
                            l.append(i['title'])
                            l.append(i['rating'])
                            l.append(i['cover'])
                            l.append(i['url'])
                            l.append(response['res']['payload']['title'])
                            all_list.append(l)
            print('\r当前爬取进度：{:.2f}%'.format(100 / 68 * a), end="")
    return all_list



if __name__ == '__main__':
    url = "https://movie.douban.com/ithil_j/activity/movie_annual2018/widget/"
    new_list = read_list(url)
    mysql = MySqlHelper()
    sql = "INSERT INTO movie_list(num, title, rating, cover, url, ranking) VALUES(%s,%s,%s,%s,%s,%s)"
    mysql.exec_many(sql, new_list)